import chardet
import re



# 获取文件原始编码格式
def detect_file_encoding(file_path):
    with open(file_path, 'rb') as f:
        raw_data = f.read(1000000)
    return chardet.detect(raw_data)['encoding']

# 转换文件编码格式为UTF-8
def encoding_file_to_utf8(source_file_path, output_file_path, batch_size):
    with open(source_file_path, 'r', encoding='GB2312', errors='ignore') as source_file, \
            open(output_file_path, 'w', encoding='utf-8') as output_file:
        # 多行读取写入
        while True:
            lines = source_file.readlines(batch_size)
            if not lines:
                break
            output_file.writelines(lines)

        source_file.close()
        output_file.close()



file_path = 'data/user_tag_query.10W.TRAIN.csv'
encoding_file_path = 'data/user_tag_query.10W.TRAIN.encoded.csv'
encoding = detect_file_encoding(file_path)
print(f"The encoding of the file is: {encoding}")

encoding_file_to_utf8(file_path, encoding_file_path, 10000)